package com.spark;

import cn.edu.hfut.dmic.contentextractor.ContentExtractor;
import cn.edu.hfut.dmic.contentextractor.News;
import com.sleepycat.je.tree.IN;
import org.apache.commons.io.FileUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.File;
import java.io.IOException;
import java.net.URLDecoder;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Created by TRS on 2017/6/26.
 */
public class BaiKe {
    public static Pattern pattern = Pattern.compile("([\\u4E00-\\u9FA5]+)");

    public static Map<String, Map<String, Integer>> baike = new HashMap<String, Map<String, Integer>>();
    public static String domain = "http://baike.baidu.com/item/";

    public static Integer nowDeep = 0;
    public static Integer deep = 3;


    public static void main(String[] args) throws IOException {

        List<String> countrys = Arrays.asList(data);
        List<String> lines = new ArrayList<String>();
        for (String c : countrys) {
            String url = domain + c;
            try {
                Map<String, Integer> maps = getRelationCountry(url);
                for (Map.Entry<String, Integer> map : maps.entrySet()) {
                    if (!c.equals(map.getKey())) {
                        lines.add(c + " " + map.getKey() + " " + map.getValue());
                    }

                    System.out.println(c + "\t" + map.getKey() + "\t" + map.getValue());
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        System.out.println(lines.size());

        File file = new File("E:\\countryRelate.txt");
        FileUtils.writeLines(file, "UTF-8", lines);


    }

    public static void loopUrl(String key) {


        try {
            String rootUrl = domain + key;
            List<String> urls = getUrls(rootUrl);
            Map<String, Integer> maps = getWords(urls);
            baike.put(key, maps);


            for (Map.Entry<String, Integer> map : maps.entrySet()) {

                System.out.println(nowDeep + " " + map.getKey() + " " + map.getValue());
                loopUrl(map.getKey());
            }

            nowDeep++;
            if (nowDeep > deep) {
                return;
            }


        } catch (Exception e) {

        }


    }

    public static Map<String, Integer> getRelationCountry(String url) throws Exception {
        String text = getTextFromUrl(url);
        Map<String, Integer> countryCount = new HashMap<String, Integer>();
        for (String c : data) {

            Integer count = 0;
            Integer start = 0;
            while ((start = text.indexOf(c, start)) != -1) {
                start += c.length();
                count++;
                System.out.println(start);
            }

            Integer recCount = countryCount.getOrDefault(c, count);

            countryCount.put(c, recCount + 1);

        }
        return countryCount;


    }


    public static String getTextFromUrl(String url) throws Exception {
        return getTextFromTHML(getHtml(url));
    }

    public static String getHtml(String url) throws Exception {
        Document doc = Jsoup.connect(url).get();
        String html = doc.html();
        return html;
    }

    private static String getTextFromTHML(String html) {
        Document doc = Jsoup.parse(html);
        String text1 = doc.text();
        // remove extra white space
        StringBuilder builder = new StringBuilder(text1);
        int index = 0;
        while (builder.length() > index) {
            Character tmp = builder.charAt(index);
            if (Character.isSpaceChar(tmp) || Character.isWhitespace(tmp)) builder.setCharAt(index, ' ');
            index += 1;
        }
        text1 = builder.toString().replaceAll(" +", " ").trim();

        return text1;
    }


    public static Map<String, Integer> getWords(List<String> urls) {
        Map<String, Integer> map = new HashMap<String, Integer>();
        for (String url : urls) {
            Matcher matcher = pattern.matcher(url);
            if (matcher.find()) {

                String key = matcher.group(1);
                Integer count = map.getOrDefault(key, 1);
                map.put(key, count + 1);

            }
        }
        return map;

    }


    public static List<String> getUrls(String url) throws IOException {
        Document doc = Jsoup.connect(url).get();

        Element content = doc.body();
        Elements links = content.getElementsByTag("a");
        List<String> urls = new ArrayList<String>();
        for (Element link : links) {
            String linkHref = link.attr("abs:href");
            if (linkHref.startsWith("http://baike.baidu.com/item/")) {
                String linkText = link.text();
                linkHref = URLDecoder.decode(linkHref, "UTF-8");
                urls.add(linkHref);
            }

        }
        return urls;
    }


    public static String[] data = {
            "中国",
            "蒙古",
            "朝鲜",
            "韩国",
            "日本",
            "越南",
            "老挝",
            "柬埔寨",
            "缅甸",
            "泰国",
            "马来西亚",
            "新加坡",
            "文莱",
            "菲律宾",
            "印度尼西亚",
            "东帝汶",
            "尼泊尔",
            "不丹",
            "孟加拉国",
            "印度",
            "斯里兰卡",
            "马尔代夫",
            "巴基斯坦",
            "阿富汗",
            "伊朗",
            "科威特",
            "沙特阿拉伯",
            "巴林",
            "卡塔尔",
            "阿联酋",
            "阿曼",
            "也门",
            "伊拉克",
            "叙利亚",
            "黎巴嫩",
            "约旦",
            "巴勒斯坦",
            "以色列",
            "塞浦路斯",
            "土耳其",
            "乌兹别克斯坦",
            "哈萨克斯坦",
            "吉尔吉斯斯坦",
            "塔吉克斯坦",
            "亚美尼亚",
            "土库曼斯坦",
            "阿塞拜疆",
            "格鲁吉亚",
            "冰岛",
            "丹麦",
            "挪威",
            "瑞典",
            "芬兰",
            "俄罗斯",
            "乌克兰",
            "白俄罗斯",
            "摩尔多瓦",
            "立陶宛",
            "爱沙尼亚",
            "拉脱维亚",
            "波兰",
            "捷克",
            "匈牙利",
            "德国",
            "奥地利",
            "列支敦士登",
            "瑞士",
            "荷兰",
            "比利时",
            "卢森堡",
            "英国",
            "爱尔兰",
            "法国",
            "摩纳哥",
            "安道尔",
            "西班牙",
            "葡萄牙",
            "意大利",
            "梵蒂冈",
            "圣马力诺",
            "马耳他",
            "克罗地亚",
            "斯洛伐克",
            "斯洛文尼亚",
            "波黑",
            "马其顿",
            "塞尔维亚",
            "黑山",
            "科索沃",
            "罗马尼亚",
            "保加利亚",
            "阿尔巴尼亚",
            "希腊",
            "埃及",
            "利比亚",
            "突尼斯",
            "阿尔及利亚",
            "摩洛哥",
            "毛里塔尼亚",
            "塞内加尔",
            "冈比亚",
            "马里",
            "布基纳法索",
            "佛得角",
            "几内亚比绍",
            "几内亚",
            "塞拉里昂",
            "利比里亚",
            "科特迪瓦",
            "加纳",
            "多哥",
            "贝宁",
            "尼日尔",
            "尼日利亚",
            "喀麦隆",
            "赤道几内亚",
            "乍得",
            "中非",
            "苏丹",
            "埃塞俄比亚",
            "吉布提",
            "索马里",
            "肯尼亚",
            "乌干达",
            "坦桑尼亚",
            "卢旺达",
            "布隆迪",
            "刚果(金)/民主刚果",
            "刚果/刚果(布)",
            "加蓬",
            "圣多美和普林西比",
            "安哥拉",
            "赞比亚",
            "马拉维",
            "莫桑比克",
            "科摩罗",
            "马达加斯加",
            "塞舌尔",
            "毛里求斯",
            "津巴布韦",
            "博茨瓦纳",
            "纳米比亚",
            "南非",
            "斯威士兰",
            "莱索托",
            "厄立特里亚",
            "澳大利亚",
            "新西兰",
            "巴布亚新几内亚",
            "所罗门群岛",
            "瓦努阿图",
            "斐济群岛/斐济",
            "基里巴斯",
            "瑙鲁",
            "密克罗尼西亚",
            "马绍尔群岛",
            "图瓦卢",
            "萨摩亚/西萨摩亚",
            "纽埃",
            "帕劳",
            "汤加",
            "加拿大",
            "美国",
            "墨西哥",
            "危地马拉",
            "伯利兹",
            "萨尔瓦多",
            "洪都拉斯",
            "尼加拉瓜",
            "哥斯达黎加",
            "巴拿马",
            "巴哈马",
            "古巴",
            "牙买加",
            "海地",
            "多米尼加共和国",
            "圣基茨和尼维斯",
            "安提瓜和巴布达",
            "多米尼克国",
            "圣卢西亚",
            "圣文森特和格林纳丁斯",
            "巴巴多斯",
            "格林纳达",
            "特立尼达和多巴哥",
            "哥伦比亚",
            "委内瑞拉",
            "圭亚那",
            "苏里南",
            "厄瓜多尔",
            "秘鲁",
            "巴西",
            "玻利维亚",
            "智利",
            "阿根廷",
            "巴拉圭",
            "乌拉圭"
    };


}
